const axios = require('axios');
const cheerio = require('cheerio');
const dotenv = require('dotenv');
const tunnel = require('tunnel');

dotenv.config();

// DeepSeek API配置
const DEEPSEEK_API_KEY = process.env.DEEPSEEK_API_KEY;
const DEEPSEEK_API_URL = process.env.DEEPSEEK_API_URL;
const DEEPSEEK_MODEL = process.env.DEEPSEEK_MODEL;

// 重试配置
const MAX_RETRIES = parseInt(process.env.MAX_RETRIES || '3', 10);
const RETRY_DELAY = parseInt(process.env.RETRY_DELAY || '2000', 10);

/**
 * 获取网页内容并支持重试
 * @param {string} url 
 * @param {Object} proxyConfig 可选的代理配置
 * @param {number} maxRetries 最大重试次数
 * @param {number} retryDelay 重试延迟(毫秒)
 * @returns {Promise<string>} HTML内容
 */
const fetchHtml = async (url, proxyConfig = null, maxRetries = 3, retryDelay = 2000) => {
  let lastError = null;
  
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      // 更丰富的浏览器请求头
      const headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.9',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'none',
        'Sec-Fetch-User': '?1',
        'Upgrade-Insecure-Requests': '1'
      };

      // 请求配置
      const requestConfig = {
        headers,
        timeout: 15000,
        maxRedirects: 5,
        validateStatus: status => status < 500 // 允许任何非500错误的状态码
      };

      // 如果提供了代理配置，添加到请求中
      if (proxyConfig && proxyConfig.enabled && proxyConfig.host && proxyConfig.port) {
        const proxyUrl = `http://${proxyConfig.host}:${proxyConfig.port}`;
        
        // 创建代理隧道
        const httpsAgent = tunnel.httpsOverHttp({
          proxy: {
            host: proxyConfig.host,
            port: parseInt(proxyConfig.port)
          }
        });
        
        // 设置代理代理
        requestConfig.httpsAgent = httpsAgent;
        requestConfig.proxy = false; // 禁用默认代理，使用httpsAgent
        
        console.log(`Using proxy tunnel for request: ${proxyUrl}`);
      }

      // 发送请求
      const response = await axios.get(url, requestConfig);
      
      // 处理常见的错误状态码
      if (response.status === 403) {
        console.warn(`Received 403 Forbidden from ${url}. The site may be blocking scrapers.`);
        throw new Error(`Access denied (403 Forbidden). The website is blocking automated access.`);
      }
      
      if (response.status === 404) {
        throw new Error(`Page not found (404). Please verify the URL.`);
      }
      
      if (response.status === 503) {
        console.warn(`Received 503 Service Unavailable from ${url}. The server might be temporarily overloaded or undergoing maintenance.`);
        throw new Error(`Service unavailable (503). The website server is temporarily unavailable. Please try again later.`);
      }
      
      if (response.status >= 400) {
        throw new Error(`Request failed with status code ${response.status}`);
      }
      
      return response.data;
    } catch (error) {
      lastError = error;
      
      // 确定是否需要重试
      const shouldRetry = attempt < maxRetries && (
        error.code === 'ECONNRESET' || 
        error.code === 'ETIMEDOUT' || 
        (error.response && (error.response.status === 503 || error.response.status === 429)) ||
        error.message.includes('503') ||
        error.message.includes('timeout')
      );
      
      if (shouldRetry) {
        console.log(`Attempt ${attempt} failed, retrying in ${retryDelay}ms...`);
        await new Promise(resolve => setTimeout(resolve, retryDelay));
        // 每次重试增加延迟时间
        retryDelay *= 1.5;
        continue;
      }
      
      // 处理常见的网络错误
      if (error.code === 'ECONNREFUSED') {
        throw new Error(`Connection refused. Please check if the URL is correct and the server is running.`);
      } else if (error.code === 'ETIMEDOUT') {
        throw new Error(`Connection timed out. The server took too long to respond.`);
      } else if (error.code === 'ENOTFOUND') {
        throw new Error(`Domain not found. Please check if the URL is correct.`);
      } else if (error.response && error.response.status === 503) {
        throw new Error(`Service unavailable (503). The website server is temporarily unavailable. Please try again later.`);
      }
      
      console.error(`Error fetching HTML from ${url}:`, error.message);
      throw new Error(`Failed to fetch webpage: ${error.message}`);
    }
  }
  
  // 如果所有重试都失败了
  throw lastError;
};

/**
 * 使用DeepSeek AI分析网页中的人物列表结构
 * @param {string} url 网页URL
 * @param {Object} proxyConfig 可选的代理配置
 * @param {Object} options 其他选项，如重试次数
 * @returns {Promise<Object>} 分析结果，包含各种选择器
 */
exports.analyzeWebpage = async (url, proxyConfig = null, options = {}) => {
  const maxRetries = options.maxRetries || MAX_RETRIES;
  const retryDelay = options.retryDelay || RETRY_DELAY;
  
  try {
    // 获取网页内容
    const html = await fetchHtml(url, proxyConfig, maxRetries, retryDelay);
    
    // 提取网页中的有效HTML结构，供AI分析使用
    const $ = cheerio.load(html);
    
    // 提取页面关键结构
    const bodyHtml = $('body').html();
    const compactHtml = simplifyHtml(bodyHtml);
    
    // 构建提示词
    const prompt = generatePrompt(url, compactHtml);
    
    // 调用DeepSeek API进行分析
    const analysisResult = await callDeepSeekApi(prompt, maxRetries, retryDelay);
    
    // 解析返回结果
    return parseAiResponse(analysisResult);
  } catch (error) {
    console.error('DeepSeek webpage analysis failed:', error);
    
    // 分类错误类型，提供更具体的错误消息
    if (error.message.includes('503') || error.message.includes('Service unavailable')) {
      throw new Error(`DeepSeek analysis failed: The target website (${url}) is temporarily unavailable (503 error). Please try again later.`);
    } else if (error.message.includes('403') || error.message.includes('Access denied')) {
      throw new Error(`DeepSeek analysis failed: The target website (${url}) is blocking our access (403 error). Try using a different proxy or contact the website administrator.`);
    } else if (error.message.includes('404')) {
      throw new Error(`DeepSeek analysis failed: The page does not exist (404 error). Please verify the URL is correct.`);
    } else if (error.message.includes('DeepSeek API key is not configured')) {
      throw new Error(`DeepSeek API key is not configured. Please add the API key to your environment variables.`);
    } else {
      throw new Error(`DeepSeek analysis failed: ${error.message}`);
    }
  }
};

/**
 * 简化HTML，便于AI处理
 * @param {string} html 
 * @returns {string} 简化后的HTML
 */
const simplifyHtml = (html) => {
  if (!html) return '';
  
  // 移除脚本和样式
  let simplified = html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
  simplified = simplified.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
  
  // 移除注释
  simplified = simplified.replace(/<!--[\s\S]*?-->/g, '');
  
  // 压缩空白
  simplified = simplified.replace(/\s{2,}/g, ' ');
  
  // 查找可能包含人员列表的部分，如果内容过长，只截取有用部分
  if (simplified.length > 20000) {
    // 查找常见的列表结构
    const personListMatches = [
      /<ul[^>]*>[\s\S]*?<\/ul>/gi,
      /<ol[^>]*>[\s\S]*?<\/ol>/gi,
      /<div[^>]*class=['"].*?list.*?['"][^>]*>[\s\S]*?<\/div>/gi,
      /<div[^>]*class=['"].*?people.*?['"][^>]*>[\s\S]*?<\/div>/gi,
      /<div[^>]*class=['"].*?member.*?['"][^>]*>[\s\S]*?<\/div>/gi,
      /<div[^>]*class=['"].*?staff.*?['"][^>]*>[\s\S]*?<\/div>/gi,
      /<div[^>]*class=['"].*?team.*?['"][^>]*>[\s\S]*?<\/div>/gi,
      /<div[^>]*class=['"].*?leadership.*?['"][^>]*>[\s\S]*?<\/div>/gi,
      /<section[^>]*>[\s\S]*?<\/section>/gi
    ];
    
    let combinedMatches = '';
    
    for (const pattern of personListMatches) {
      const matches = simplified.match(pattern);
      if (matches && matches.length > 0) {
        combinedMatches += matches.join('\n');
      }
    }
    
    if (combinedMatches.length > 1000) {
      simplified = combinedMatches;
    } else {
      // 如果没有找到明显的列表，取前20000字符
      simplified = simplified.substring(0, 20000);
    }
  }
  
  return simplified;
};

/**
 * 生成提示词
 * @param {string} url 
 * @param {string} html 
 * @returns {string} 完整提示词
 */
const generatePrompt = (url, html) => {
  return `您是一位专业的网页结构分析专家，特别善于分析军事和政府网站上的人员列表页面结构。请分析以下来自 ${url} 的HTML，找出包含人员信息的列表部分，并提供最准确的CSS选择器。

HTML内容:
\`\`\`html
${html}
\`\`\`

请注意，军事网站如PACAF.af.mil通常有多种不同格式的人员列表，可能包括：
1. 图片卡片式布局的人员列表
2. 表格式的人员列表
3. 简单链接列表形式的人员目录
4. 分层级的领导层结构展示

请分析并提供以下CSS选择器:
1. listItemSelector - 列表中每个人员的容器元素选择器（可能是多种格式，请提供逗号分隔的多个选择器）
2. nameSelector - 人员名称的元素选择器，相对于listItemSelector
3. positionSelector - 人员职位的元素选择器，相对于listItemSelector
4. imageSelector - 人员图片的元素选择器，相对于listItemSelector
5. detailLinkSelector - 指向人员详情页的链接选择器，相对于listItemSelector
6. rankSelector - 人员军衔的元素选择器，相对于listItemSelector（如美国空军的General, Lieutenant General等）
7. organizationSelector - 人员所属组织/单位的元素选择器，相对于listItemSelector

军事网站常见的选择器模式包括：
- 详情页链接可能是：a[href*="/Biographies/"], a[href*="/bio/"], a[href*="/biography/"]
- 图片可能是：img[src*="bio"], img[src*="portrait"], .official-photo
- 职位和军衔常在同一元素中，如："General John Doe, Commander"

只返回一个JSON格式的对象，包含以上所有选择器。例如:
{
  "selectors": {
    "listItemSelector": ".bio-card, .bio-listing, .leader-item, table tr[valign='top'], .gridspan",
    "nameSelector": ".name, .bio-name, h2, h3, a[href*='/bio/'] strong",
    "positionSelector": ".position, .bio-title, .duty-title, p",
    "imageSelector": "img.bio-photo, img[src*='bio'], .official-photo",
    "detailLinkSelector": "a, a[href*='/Biographies/'], a[href*='/bio/'], .name a",
    "rankSelector": ".rank, span[class*='rank']",
    "organizationSelector": ".org, .unit, .command"
  }
}

请确保选择器尽可能准确简洁。如果某个选择器未找到，返回空字符串。不要包含任何解释或说明，只返回JSON对象。`;
};

/**
 * 调用DeepSeek API，支持重试
 * @param {string} prompt 
 * @param {number} maxRetries 最大重试次数
 * @param {number} retryDelay 重试延迟(毫秒)
 * @returns {string} API返回的结果
 */
const callDeepSeekApi = async (prompt, maxRetries = MAX_RETRIES, retryDelay = RETRY_DELAY) => {
  if (!DEEPSEEK_API_KEY) {
    throw new Error('DeepSeek API key is not configured.');
  }
  
  let lastError = null;
  
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const response = await axios.post(
        DEEPSEEK_API_URL,
        {
          model: DEEPSEEK_MODEL,
          messages: [
            {
              role: 'system',
              content: 'You are a webpage structure analyzer specialized in detecting people lists and extracting CSS selectors.'
            },
            {
              role: 'user',
              content: prompt
            }
          ],
          temperature: 0.1,
          max_tokens: 4096
        },
        {
          headers: {
            'Content-Type': 'application/json',
            'Authorization': `Bearer ${DEEPSEEK_API_KEY}`
          },
          timeout: 600000 // 600秒超时
        }
      );
      
      return response.data.choices[0].message.content;
    } catch (error) {
      lastError = error;
      
      // 确定是否需要重试
      const shouldRetry = attempt < maxRetries && (
        error.code === 'ECONNRESET' || 
        error.code === 'ETIMEDOUT' || 
        (error.response && (error.response.status === 429 || error.response.status >= 500 && error.response.status < 600)) ||
        error.message.includes('timeout')
      );
      
      if (shouldRetry) {
        console.log(`DeepSeek API call attempt ${attempt} failed, retrying in ${retryDelay}ms...`);
        await new Promise(resolve => setTimeout(resolve, retryDelay));
        // 每次重试增加延迟时间
        retryDelay *= 1.5;
        continue;
      }
      
      console.error('DeepSeek API call failed:', error.response?.data || error.message);
      throw new Error(`DeepSeek API call failed: ${error.message}`);
    }
  }
  
  // 如果所有重试都失败了
  throw lastError;
};

// 将callDeepSeekApi函数添加到exports中
exports.callDeepSeekApi = callDeepSeekApi;

/**
 * 解析AI返回的文本为结构化对象
 * @param {string} apiResponse 
 * @returns {Object} 解析后的结果
 */
const parseAiResponse = (apiResponse) => {
  try {
    // 尝试从响应中提取JSON部分
    const jsonMatch = apiResponse.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      const jsonString = jsonMatch[0];
      const result = JSON.parse(jsonString);
      
      // 检查结果是否包含预期的结构
      if (!result.selectors) {
        result.selectors = {
          listItemSelector: '',
          nameSelector: '',
          positionSelector: '',
          imageSelector: '',
          detailLinkSelector: '',
          rankSelector: '',
          organizationSelector: ''
        };
      }
      
      return result;
    }
    
    // 如果无法匹配JSON，返回空结果
    return {
      selectors: {
        listItemSelector: '',
        nameSelector: '',
        positionSelector: '',
        imageSelector: '',
        detailLinkSelector: '',
        rankSelector: '',
        organizationSelector: ''
      }
    };
  } catch (error) {
    console.error('Failed to parse DeepSeek response:', error);
    return {
      selectors: {
        listItemSelector: '',
        nameSelector: '',
        positionSelector: '',
        imageSelector: '',
        detailLinkSelector: '',
        rankSelector: '',
        organizationSelector: ''
      }
    };
  }
};

/**
 * 分析详情页的结构和内容
 * @param {string} url 详情页URL
 * @param {Object} proxyConfig 代理配置
 * @returns {Promise<Object>} 分析结果
 */
exports.analyzePersonnelDetail = async (url, proxyConfig = null) => {
  try {
    // 获取网页内容
    const html = await fetchHtml(url, proxyConfig);
    
    // 构建提示词
    const prompt = `您是一位专业的军事人员档案分析专家。请分析以下来自 ${url} 的美国军事人员详情页内容，提取该人员的所有关键信息。

详情页内容:
\`\`\`html
${html.substring(0, 15000)}
\`\`\`

请特别注意以下美国军事特有的信息格式：
1. 军衔格式如：General, Lieutenant General, Major General, Brigadier General, Colonel等
2. 教育经历通常包括军事院校如：Air War College, Air Command and Staff College, Squadron Officer School等
3. 军事人员简历通常按时间倒序排列，包含服役日期、职位和单位
4. 战斗部署经历(Deployments)可能单独列出
5. 飞行小时数和飞行机型可能会特别提及
6. 军事奖章和荣誉通常在结尾列出

请分析并以JSON格式提供以下信息:
1. name - 人员姓名/全名
2. currentPosition - 当前职位/岗位
3. rank - 军衔/级别
4. organization - 所属组织/单位
5. education - 教育背景（数组）
6. career - 职业经历（数组，按时间倒序）
7. deployments - 战斗部署经历（数组，如果有）
8. flightInfo - 飞行信息，包括小时数和机型（如果有）
9. awards - 军事奖章和荣誉（数组）
10. effectiveDates - 任命生效日期（如果有）
11. summary - 简要概述/简介

例如:
{
  "name": "Gen. Kenneth S. Wilsbach",
  "currentPosition": "Commander, Pacific Air Forces",
  "rank": "General",
  "organization": "Pacific Air Forces",
  "education": ["Bachelor of Science degree in computer engineering, University of Florida", "Air War College, Maxwell AFB"],
  "career": ["July 2020–Present, Commander, Pacific Air Forces", "July 2018–July 2020, Commander, U.S. Air Forces Central Command"],
  "deployments": ["Operation Iraqi Freedom", "Operation Enduring Freedom"],
  "flightInfo": "The general is a command pilot with more than 5,000 flying hours in the F-15C, F-16A/B/C/D, F-22A, MC-12 and T-38A",
  "awards": ["Distinguished Service Medal", "Defense Superior Service Medal"],
  "effectiveDates": "July 2020",
  "summary": "Gen. Kenneth S. Wilsbach is the Commander, Pacific Air Forces; Air Component Commander for U.S. Indo-Pacific Command; and Executive Director, Pacific Air Combat Operations Staff, Joint Base Pearl Harbor-Hickam, Hawaii."
}

请确保尽可能提取所有可用信息。如果某项信息不存在，请使用空字符串或空数组。只返回JSON对象，不要包含任何解释或说明。`;

    // 调用DeepSeek API分析
    const analysisResult = await callDeepSeekApi(prompt);
    
    try {
      // 尝试从响应中提取JSON部分
      const jsonMatch = analysisResult.match(/\{[\s\S]*\}/);
      if (jsonMatch) {
        return JSON.parse(jsonMatch[0]);
      }
    } catch (error) {
      console.error('解析DeepSeek返回的详情页分析结果失败:', error);
    }
    
    return {
      error: '无法解析详情页内容'
    };
  } catch (error) {
    console.error('详情页分析失败:', error);
    return {
      error: error.message
    };
  }
}; 